package com.curry.apiprovider.crawler;

import com.curry.apiprovider.crawler.pageProcessor.MyPipLine;
import lombok.Data;
import org.springframework.stereotype.Component;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.pipeline.ConsolePipeline;
import us.codecraft.webmagic.pipeline.Pipeline;
import us.codecraft.webmagic.processor.PageProcessor;

import java.util.List;
import java.util.Map;

@Data
@Component
public class PageProcessorFactory {

    private PageProcessor pageProcessor;
    private MyPipLine pipeline;
    private String  data ;
    public void startSpider(String url, int threadCount) {
        Spider.create(pageProcessor)
                .addUrl(url).
                addPipeline(new ConsolePipeline())
                .addPipeline(pipeline)
                .thread(threadCount)
                //启动爬虫
                .run();
        data = pipeline.getData();
    }


}
